from requests_html import HTMLSession
import os

if __name__=="__main__":
    url = "http://www.biquger.com"
    session = HTMLSession()
    print('start...')
    if not os.path.exists("xiaoshuo"):
        os.mkdir("xiaoshuo")

def download(chapt,novel,catetory):
        chapt_res = session.get(chapt['url'])
        booktext = chapt_res.html.find("div#booktext")[0].text
        booktext_list = "\n".join(booktext.splitlines()[:-4])
        file_name = "xiaoshuo/"+catetory+"/"+novel+".txt"
        with open(file=file_name,encoding="utf-8",errors='ignore',mode="a+") as xf:
            xf.write(booktext_list)


def download_novel(url,novel):
    res = session.get(url=url)
    chapt_list = res.html.xpath("//div[@id='list']/dl/dd")
    all_chapt = []
    catetory = res.html.xpath("//div[@class='con_top']/a[2]/text()")[0]
    dir = "xiaoshuo/"+catetory
    if not os.path.exists(dir):
        os.mkdir(dir)
    if not os.path.exists(dir):
        os.mkdir(dir)
    for chapt in chapt_list:
        title = chapt.xpath("//a/@title")[0]
        href = chapt.xpath("//a/@href")[0]
        all_chapt.append({
            "title":title,
            "url":href
        })
    for chapt in all_chapt:
        download(chapt=chapt,novel=novel,catetory=catetory)
        print(catetory,novel,chapt,"done...")

nav_urls = ["http://www.biquger.com/category/1_1.html","http://www.biquger.com/category/2_1.html","http://www.biquger.com/category/3_1.html","http://www.biquger.com/category/4_1.html","http://www.biquger.com/category/6_1.html","http://www.biquger.com/category/7_1.html","http://www.biquger.com/category/8_1.html","http://www.biquger.com/category/10_1.html"]
for url in nav_urls:
    nav_res = session.get(url)
    total = nav_res.html.xpath("//div[@id='pagelink']/a[last()]/text()")[0]
    prefix_url = url.split("_")[0]
    for i in range(1,int(total)+1):
        suffix_url = "_"+str(i)+".html"
        next_url = prefix_url+suffix_url
        next_res = session.get(next_url)
        novel_list = next_res.html.xpath("//div[@id='newscontent']/div[@class='l']/ul/li")
        for novel in novel_list:
            title = novel.xpath("//span[@class='s2']/a/text()")[0]
            href = novel.xpath("//span[@class='s2']/a/@href")[0]
            download_novel(url=href,novel=title)






